package com.xiaotu.spider.project.pageprocessor.news;

import com.xiaotu.common.aop.ALogAspect;
import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.util.*;
import com.xiaotu.spider.AUrlPageProcessor;
import net.sf.json.JSONObject;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;

import java.util.*;
import java.util.stream.Stream;


/**
 * 爬取新浪新闻数据
 */
@Component("NewsSinaPageProcessor")
public class NewsSinaPageProcessor extends AUrlPageProcessor
{

    private static final Logger LOGGER = LoggerFactory.getLogger(ALogAspect.class);

    private static final String BASE_URL = "http://cre.mix.sina.com.cn/api/v3/get";

    private static final String PARAM_KEY = "REQ_PARAM_KEY";

    private static final String[] PARAM_KEYS = new String[]{"callback", "cateid", "cre", "mod", "merge", "statics",
            "length", "up", "down", "fields", "tm", "action", "top_id", "offset", "ad", "_"};

    private static final String[] SUB_PARAM_KEYS = (String[]) ArrayUtils.addAll(PARAM_KEYS, new String[]{"port",
            "imp_type", "pcProduct", "ctime"});

    private static final String[] CATE_IDS = new String[]{"1Q"};

    private static final int MAX_PAGE = 50;

    private String filePath;


    @Override
    public void process(Page page)
    {
        if (page.getUrl().toString().contains("api/v3/get"))
            this.listProcess(page);
        else
            this.detailProcess(page);
    }

    /**
     * 处理页面内容
     *
     * @param page 页面
     */
    private void detailProcess(Page page)
    {
        String content = page.getHtml().xpath("//div[@id='artibody']").get();
        if (StringUtils.isEmpty(content))
            return;
        content = HtmlUtil.delHTMLTag(content, false);
        if (StringUtils.isEmpty(content))
            return;
        String fileName = page.getUrl().toString();
        fileName = fileName.substring(fileName.lastIndexOf(SepratorUtil.SEP_SLASH_EN) + SepratorUtil.SEP_SLASH_EN
                .length());
        fileName = fileName.substring(0, fileName.indexOf("?") > 0 ? fileName.indexOf("?") : fileName.length());
        fileName = this.filePath + fileName;
        if (!FileUtil.fileExits(fileName))
        {
            FileUtil.writeFile(fileName, content);
            LOGGER.debug(fileName + " writed success!");
        }
        else
            LOGGER.debug(fileName + " exists!");
    }

    /**
     * 处理列表
     *
     * @param page 页面
     */
    private void listProcess(Page page)
    {
        String data = page.getRawText();
        if (StringUtils.isEmpty(data))
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        int start = data.indexOf("{");
        int end = data.lastIndexOf("}") + "}".length();
        if (start < 0 || end < 1 || end < start)
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        data = data.substring(start, end);
        Map<String, Object> resMap = GsonUtils.fromJson(data, Map.class);
        LOGGER.debug("response data:" + data);
        List<Map<String, Object>> dataList = (List<Map<String, Object>>) resMap.get("data");
        if (dataList == null || dataList.isEmpty())
            throw new SpiderException(SpiderExceptionCode.DataInvalideException);
        Map<String, String> paraMap = (Map<String, String>) page.getRequest().getExtra(PARAM_KEY);
        dataList.stream().filter(newsMap -> newsMap.get("url") != null && !StringUtil.nullToString(newsMap.get("url")
        ).contains("/slide_")).forEach(newsMap ->
        {
            String time = "";
            if (newsMap.get("ctime") != null)
            {
                time = DateUtils.getDateString(DateUtils.transferLongToDate(DateUtils.SECONDS_FORMAT, MathUtil
                        .toBigDecimal(newsMap.get("ctime")).longValue() * 1000), DateUtils.SECONDS_FORMAT);
            }
            String title = newsMap.get("title") == null ? (newsMap.get("ltitle") == null ? newsMap.get("mintro") + ""
                    : newsMap.get("ltitle") + "") : newsMap.get("title") + "";
            String line = time + "=====" + title;
            LOGGER.debug("[" + paraMap.get("cateid") + "]news item:" + line);
            PageUtils.addTargetRequest(page, StringUtil.nullToString(newsMap.get("url")), page.getRequest().getExtras
                    ());
        });

        if (CATE_IDS[0].equals(paraMap.get("cateid")))//娱乐大类
        {
            paraMap.put("req_time", MathUtil.toBigDecimal(resMap.get("req_time")).longValue() + "");
            int pageNo = this.setUrlAndParam(paraMap, CATE_IDS[0]);
            if (pageNo > MAX_PAGE)
                return;
            PageUtils.addTargetRequest(page, paraMap.get(URL_KEY), BeanUtils.clone((HashMap<String, Object>) page
                    .getRequest().getExtras()), true);

            //将子类添加到爬取任务中
            Stream.of(CATE_IDS).skip(1).forEach(cateId -> this.addSubNewsLink(page, cateId, pageNo));
        }
    }

    /**
     * 添加娱乐下的子类链接
     *
     * @param page   页面
     * @param cateId 子类id
     * @param pageNo 页码
     */
    private void addSubNewsLink(Page page, String cateId, int pageNo)
    {
        Map<String, String> paraMap = (Map<String, String>) page.getRequest().getExtra(PARAM_KEY);
        paraMap.put("up", String.valueOf(pageNo - 2));
        this.setUrlAndParam(paraMap, cateId);
        PageUtils.addTargetRequest(page, paraMap.get(URL_KEY), BeanUtils.clone((HashMap<String, Object>) page
                .getRequest().getExtras()), true);
    }

    /**
     * 封装请求参数
     *
     * @param paraMap 请求参数缓存
     * @param cateid  分类id
     * @return 页码
     */
    private int setUrlAndParam(Map<String, String> paraMap, String cateid)
    {
        boolean isSub = !CATE_IDS[0].equals(cateid);//是否为子类新闻
        int up = paraMap.containsKey("up") ? Integer.parseInt(paraMap.get("up")) + 1 : 0;
        String timestamp = String.valueOf(System.currentTimeMillis());
        paraMap.put("callback", "jQuery1112024109137048937557_" + timestamp);
        paraMap.put("cateid", cateid);

        paraMap.put("cre", "tianyi");

        //        paraMap.put("mod", "pcent");
        paraMap.put("mod", isSub ? "pcentc" : "pcent");

        paraMap.put("merge", "3");
        paraMap.put("statics", "1");
        paraMap.put("length", "15");
        paraMap.put("up", String.valueOf(up));
        paraMap.put("down", "0");
        paraMap.put("fields", "media,author,labels_show,commentid,comment_count,title,ltitle,url,info,thumbs,thumb," +
                "ctime,reason,video_id,hotTag,img_count,gif,live_stime,live_etime,media_id,summary,orgUrl,show,intro," +
                "docid,playtimes,video_height,video_width,time_length,user_icon,uid");
        paraMap.put("tm", paraMap.containsKey("req_time") ? paraMap.get("req_time") : timestamp.substring(0, 10));
        paraMap.put("action", paraMap.containsKey("action") ? "1" : "0");
        paraMap.put("top_id", "24lrw,24qBY,24lrw,24m53,24uEt,24now,24ttQ,24rCS,24jrj,24jrj,24ic4,24sVZ,24Pwz");
        paraMap.put("offset", "0");
        paraMap.put("ad", "{\"rotate_count\":100,\"platform\":\"pc\",\"channel\":\"tianyi_pcent\"," +
                "\"page_url\":\"http://ent.sina.com.cn/\",\"timestamp\":" + timestamp + "}");
        paraMap.put("_", timestamp);


        if (isSub)
        {
            paraMap.put("port", "3602");
            paraMap.put("imp_type", "2");
            paraMap.put("pcProduct", "2");
            paraMap.put("ctime", timestamp.substring(0, 10));
        }


        StringBuffer buffer = new StringBuffer("?");
        Stream.of(isSub ? SUB_PARAM_KEYS : PARAM_KEYS).forEach(key ->
        {
            Object val = paraMap.get(key);
            buffer.append(key + SepratorUtil.SEP_EQUAL + KeywordEncode.encode(val instanceof JSONObject ? GsonUtils
                    .toJson(val) : val.toString()) + SepratorUtil.SEP_URL_PARAM);
        });
        paraMap.put(URL_KEY, BASE_URL + buffer.substring(0, buffer.length() - SepratorUtil.SEP_URL_PARAM.length()));
        return up;
    }

    @Override
    protected List<JSONObject> getStartRequestUrlsAndParams()
    {
        JSONObject target = new JSONObject();
        Map<String, String> paraMap = new HashMap<>();
        this.setUrlAndParam(paraMap, CATE_IDS[0]);
        target.put(PARAM_KEY, paraMap);
        target.put(URL_KEY, paraMap.get(URL_KEY));
        List<JSONObject> list = new ArrayList<>();
        list.add(target);
        return list;
    }

    @Override
    public void initRunParams(Map<String, Object> map)
    {
        String path = StringUtil.nullToString(map.get("path"));
        if (StringUtils.isEmpty(path))
            throw new SpiderException(SpiderExceptionCode.PageProcessorParameterErrorException);
        this.filePath = path;
    }

    @Override
    protected String[] getStartRequestUrls()
    {
        return null;
    }

    @Override
    protected String getSpiderType()
    {
        return "NewsSina";
    }

    @Override
    public String getPageDataType(Page page)
    {
        return Constants.PageDataType.SEARCH_LIST;
    }
}
